'''
Created on Jan 2011

@author: oliver

Version 1.0
'''

import numpy as np
import sys
import tables as hdf5_table
import datetime
import numpy as np

MISSING_VALUE_OUTPUT = -999

class XpyTable:
    #_data array with variable names (columns)
    #tab= XpyTable()
    #tab.add_variable('type')
    #tab.add_row()
    #tab._data[1, arr.cx('type')] = 2  


    def __init__(self, numpy_data=None, variable_name_list=None):
        """ numpy_data: must be numpy_array, variable_name_list must be list"""
        if numpy_data is None:
            self._data = np.ones((0, 0))
            self.varnames = []
        else:
            if variable_name_list is None:#
                variable_name_list = []
                for x in range(0, numpy_data.shape[1]):
                    variable_name_list.append("var_{0}".format(x))
            if numpy_data.shape[1] != len(variable_name_list):
                print "Error: _data and variable names do not fit"
                sys.exit()
            self._data = numpy_data
            self.varnames = variable_name_list

    def __repr__(self):
        return  "*{0}*\n{1}".format(self.varnames, self._data)

    def add_variable(self, variable_name, default_value=np.nan):
        """adds a variable colunm"""
        if (self.n_variables == 0):
            self._data = np.ones((0, 1))
        else:
            col = np.ones(self.n_rows) * default_value
            self._data = np.c_[self._data, col]
        self.varnames.append(variable_name)

    def add_row(self, value=np.nan):
        """adds a row with _data
            extends variables if required
            adds NaN if row too short or values not set"""
        value = np.array(value).squeeze()
        d = self.n_variables - value.size
        if (d == 0):
            row = np.ones((1, self.n_variables)) * value
            self._data = np.r_[self._data, row]
        elif d < 0:
            for x in range(self.n_variables + 1, self.n_variables + 1 + abs(d)):# make variables
                self.add_variable("var_{0}".format(x))
            self.add_row(value)
        elif d > 0:
            for x in range(0, d):
                value = np.append(value, np.NaN)
            self.add_row(value)

    @property
    def n_rows(self):
        return self._data.shape[0]

    @property
    def n_variables(self):
        return len(self.varnames)

    @property
    def data(self):
        return self._data

    def column(self, varname, row_indices=None):
        if row_indices is None:
            return self._data[:, self.cx(varname)]
        else:
            return self._data[row_indices, self.cx(varname)]

    def row(self, row_indices):
        return self._data[row_indices, :]

    def cx(self, varname):
        """returns column index the variable and np.nan if not found """
        cnt = 0
        for v in self.varnames:
            if v == varname:
                break
            cnt = cnt + 1
        if (cnt >= self.n_variables):
            cnt = np.nan
        return cnt

    def delete_rows(self, idx):
        self._data = np.delete(self._data, idx, 0)

    def delete_variable(self, varname):
        c = self.cx(varname)
        if (c >= 0):
            self._data = np.delete(self._data, c, 1)
            self.varnames.remove(varname)

    def find_equals(self, variable_name, value):
        return np.nonzero(self._data[:, self.cx(variable_name)] == value)[0]

    def find_unequals(self, variable_name, value):
        return np.nonzero(self._data[:, self.cx(variable_name)] != value)[0]

    def find_smaller(self, variable_name, value):
        return np.nonzero(self._data[:, self.cx(variable_name)] < value)[0]

    def find_larger(self, variable_name, value):
        return np.nonzero(self._data[:, self.cx(variable_name)] > value)[0]

    def save_to_hdf5(self, filename, dataname):
        #make record
        ty = []
        for x in self.varnames:
            ty.append((x, self._data.dtype))
        y = np.core.records.array(self._data, dtype=ty)

        h5file = hdf5_table.openFile(filename, mode="a", title="Expy _data")
        try:
            h5file.removeNode("/", dataname, True)
        except:
            pass
        h5file.createTable("/", dataname, y)
        h5file.flush()
        h5file.close()

    def col_mean(self, variable_name):
        """ return nan_mean of a variable"""
        return np.mean(remove_nan( self._data[:, self.cx(variable_name)] ))

    def col_median(self, variable_name):
        """ return nan_median of a variable"""
        return np.median(remove_nan( self._data[:, self.cx(variable_name)] ))

    def col_std(self, variable_name):
        """ return nan_std of a variable"""
        return np.std(remove_nan( self._data[:, self.cx(variable_name)] ))



def load_hdf5_table(filename, tablename):
    """ hdf5_table(self, filename, tablename):
        loads datatable from hdf5 file and returns XpyTable """
    h5file = hdf5_table.openFile(filename)
    d = eval("h5file.root.{0}".format(tablename))

    arr = np.empty((d.nrows, len(d.colnames))) * np.NaN
    print "XpyTable shape: {0}".format(arr.shape)
    cnt = 0
    for var in d.colnames:
        print "Var: ", var
        arr[:, cnt] = d.col(var)
        cnt = cnt + 1

    tab = XpyTable(arr, d.colnames)
    h5file.close()
    return tab


def read_text_data(filename, varnames_in_1st_line=True, delimiter=',', comment_tag='', data_tag=''):
    """Reads all data lines from text file
       data_tag: read in only lines with starts with a <data_tag> 
       comment_tag: ignores lines that start with <comment_tag> 
       returns data, and varnames as string arrays and ignored_text
    """
    print "reading ", filename
    f = open(filename, 'r')
    varnames = []
    data = []
    ignored_text = ""
    n = 0
    last = -9
    for line in f:
        if (len(comment_tag) == 0 or not line.startswith(comment_tag)) and (len(data_tag) == 0 or line.startswith(data_tag)) :
            line = line.rstrip().lstrip()
            if n == 0 and varnames_in_1st_line:
                varnames.append([ str(x) for x in line[len(data_tag):].split(delimiter) ])
            else:
                data.append([ str(x) for x in line[len(data_tag):].split(delimiter) ])
                if (last < 0):
                    last = len(data[-1])
                else:
                    if (last != len(data[-1])):
                        print "Error in line {0}: {1} columns".format(n + 1, len(data[-1]))
            n = n + 1
        else:
            ignored_text = ignored_text + line

    if len(varnames) > 0:
        varnames = varnames[0]
    print "- data: ", len(data), "x", len(data[0])
    print "- varnames: ", len(varnames)
    f.close()
    return (data, varnames, ignored_text)

def _make_varnames(n):
    varnames = []
    for i in range(n):
        varnames.append('var_{0}'.format(i + 1))
    return varnames

def np_data_to_text(filename, data, varnames=[], delimiter=',', missingvalue=MISSING_VALUE_OUTPUT):
    """export numpy data to text file"""

    #make sure data is nparray and replace missings
    data = np.array(data)
    idx = np.isnan(data)
    data[idx] = missingvalue
    #Give variable names if number varnames does not match number of variables
    if len(varnames) != data.shape[1]:
        varnames = _make_varnames(data.shape[1])

    f = open(filename, 'w')

    #add varnames
    for x in varnames:
        f.write('{0}{1}'.format(x, delimiter))
    f.write("\n")
    #add data
    for r in data:
        for c in r:
            f.write('{0}{1}'.format(c, delimiter))
        f.write("\n")


def np_data_to_spss(filename, data, varnames=[], missingvalue=MISSING_VALUE_OUTPUT, commenttext=[], labels=[]):
    """use exp2SPSS to export data to an SPSS Syntax File. 
    
        
    <filename (obligatory)>
     Output Filename (string). The output file is entitled <filename>.data.sps .
    <data (obligatory)>
     the 2d-numpy array to be exported to an SPSS Syntax File (array or list).
     ndarray: columns represent variables
     List: the lists within the list are the rows.
    <varnames> 
     can include the variable names (list of strings)
     <labels> 
     can include the labels of the variables (list of strings)
    <MissingValue>
     missing value (int)
    <commenttext>
     adds comments to file
     
     based on code of Roel Boursards
     """

    #print      
    print 'spss sytax file will be saved as {0}'.format(filename)

    #make sure data is ndarray and replace missings
    data = np.array(data)
    idx = np.isnan(data)
    data[idx] = missingvalue
    #Give variable names if number varnames does not match number of variables
    if len(varnames) != data.shape[1]:
        varnames = _make_varnames(data.shape[1])

    f = open(filename, 'w')
    f.write('*  This SPSS syntax file is generated with Expy_analysis at {0}\n'.format(datetime.datetime.now()))
    f.write('*  Run it in SPSS syntax editor (Run/All) to import the containing data.\n')

    #add comments    
    for x in commenttext:
        f.write('* {0}\n'.format(x))
    f.write('\nDATA LIST FREE\n / ')
    #add varnames
    for x in varnames:
        f.write(' {0}'.format(x))
    #add data
    f.write('.\nBEGIN DATA.\n')
    for x in data.flat:
        f.write('\n{0}'.format(x))
    f.write('\nEND DATA.\n')
    #add lables if number of labels equals number of variables
    if len(labels) == data.shape[1]:
        for i in range(len(labels)):
            f.write("VARIABLE LABELS {0} '{1}' .\n".format(varnames[i], labels[i]))
    #add info about missingvalues    
    f.write('MISSING VALUES all( {0} ).\n\n'.format(missingvalue))
    #finish syntax
    for x in varnames:
        f.write('VARIABLE LEVEL {0} (scale).\n'.format(x))

    f.close()
    

def _str_to_numpy_number(num_str):
    """
    returns numpy.int32 or numpy.float32 and np.isnan not convertable to number (catches exceptions)

    tries to produce int32 data for a parsimonious handling off large integer data set 
    """
    if num_str.find(".") >= 0:
        try:
            rtn = np.float64(num_str)
            return rtn
        except:
            return np.nan
    else:
        try:
            rtn = np.int32(num_str)
            return rtn
        except:
            return np.nan

def str_array_to_numpy_array(str_data):
    """converts string 2d-string array into 2d numpy array"""
    np_data = []
    for line in str_data:
        tmp = []
        for x in line:
            tmp.append(_str_to_numpy_number(x))
        np_data.append(tmp)
    print "numpy array ", len(np_data), "x", len(np_data[0])
    return np.array(np_data)



def remove_nan(data):
    """squeezes and removes all nans"""
    data = np.squeeze(data)
    data = np.append(data, np.nan) # workaround to avoid "one element arrays" (isnan return in this case only boolean and not an array)
    idx = np.nonzero(np.isnan(data))
    data = np.delete(data, idx[0])
    return data

def get_values(array):
    """returns all values (ones) in numpy array """
    x = np.sort(array)
    values = [x[0]]
    last = x[0]
    for v in x:
        if v != last:
            values.append(v)
            last = v
    return np.array(values)
